spb/social-runtime-crawler

1"use client";23import { use } from "react";4import Link from "next/link";5import { useApi } from "@/lib/api";6import { fmtDate } from "@/lib/format";7import { Bar, Empty, Metric, Panel, Skeleton, Table, Tag } from "@/components/ui";89interface Detail {10  platform: string;11  summary: { confidence: number; page_types: number; entity_types: number; navigation_actions: number; network_schemas: number; media_patterns: number; sessions: number } | null;12  model: {13    updated_at: string;14    response_patterns: Record<string, { shape_hash: string; hostname: string; path_pattern: string; method: string; graphql_operation?: string; likely_entity_types: Record<string, number>; observed_count: number; entity_yield_total: number; confidence: number; triggered_by: Record<string, number>; first_seen: string; last_seen: string }>;15    action_patterns: Record<string, { action_type: string; from_page_type: string; observed_count: number; avg_new_entities: number; avg_network_responses: number; to_page_types: Record<string, number>; failures: number }>;16    page_types: Record<string, { page_type: string; observed_count: number; url_patterns: Record<string, number>; avg_entities: number; entity_types: Record<string, number>; avg_confidence: number }>;17    media_patterns: Record<string, { hostname: string; kind: string; observed_count: number }>;18    sessions_learned: string[];19  } | null;20  manifest: { compiled_at: string; confidence: number; network_patterns: unknown[] } | null;21  profiles: { alias: string; has_state: boolean }[];22}2324export default function PlatformPage({ params }: { params: Promise<{ platform: string }> }) {25  const { platform } = use(params);26  const { data, loading } = useApi<Detail>(`/platforms/${platform}`, { refreshMs: 15_000 });27  if (loading) return <Skeleton rows={10} />;28  if (!data) return <Empty>unknown platform</Empty>;29  const m = data.model;30  const rps = m ? Object.values(m.response_patterns).filter((r) => Object.keys(r.likely_entity_types).length).sort((a, b) => b.confidence - a.confidence) : [];31  const aps = m ? Object.values(m.action_patterns).filter((a) => a.from_page_type !== "ANY").sort((a, b) => b.observed_count - a.observed_count) : [];32  const pts = m ? Object.values(m.page_types).sort((a, b) => b.observed_count - a.observed_count) : [];33  return (34    <div className="space-y-4">35      <div className="text-xs text-dim mono">36        <Link href="/platforms" className="hover:text-cyan">connectors</Link> / {platform}37      </div>38      <div className="flex flex-wrap items-end gap-4">39        <h1 className="text-2xl font-semibold tracking-tight capitalize">{platform}</h1>40        {data.summary && (41          <div className="ml-auto flex items-center gap-3 w-72">42            <span className="text-xs text-dim">confidence</span>43            <Bar value={data.summary.confidence / 100} className="flex-1" color={data.summary.confidence >= 70 ? "#43e69a" : "#38e1ff"} />44            <span className="mono">{data.summary.confidence}%</span>45          </div>46        )}47      </div>48      {!m ? (49        <Panel>50          <Empty>51            Nothing learned yet on {platform}. {["youtube", "reddit"].includes(platform) ? "Launch a learning mission from the Missions page." : "No adapter yet — this platform is scheduled for a later phase."}52          </Empty>53        </Panel>54      ) : (55        <>56          <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">57            <Panel title="Learned model">58              <Metric label="page types" value={data.summary?.page_types} />59              <Metric label="entity types" value={data.summary?.entity_types} />60              <Metric label="navigation actions" value={data.summary?.navigation_actions} />61              <Metric label="network schemas" value={data.summary?.network_schemas} />62              <Metric label="media patterns" value={data.summary?.media_patterns} />63              <Metric label="sessions learned from" value={m.sessions_learned.length} />64              <Metric label="updated" value={fmtDate(m.updated_at)} />65              <Metric label="compiled manifest" value={data.manifest ? `${fmtDate(data.manifest.compiled_at)} · ${data.manifest.network_patterns.length} patterns` : "not yet (needs ≥ 40 %)"} />66              <Metric label="browser profiles" value={data.profiles.length ? data.profiles.map((p) => `${p.alias}${p.has_state ? " ✓" : ""}`).join(", ") : "none — run pnpm login"} />67            </Panel>68            <Panel title="Page types" sub="what the crawler recognises, and what each page usually yields" className="xl:col-span-2" pad={false}>69              <div className="p-4">70                <Table head={["page type", "seen", "avg entities", "avg conf", "url patterns", "entity mix"]} dense>71                  {pts.map((p) => (72                    <tr key={p.page_type}>73                      <td>74                        <Tag color="#38e1ff">{p.page_type}</Tag>75                      </td>76                      <td className="mono">{p.observed_count}</td>77                      <td className="mono">{p.avg_entities.toFixed(1)}</td>78                      <td className="mono">{p.avg_confidence.toFixed(2)}</td>79                      <td className="mono text-[11px] text-fg-2">80                        {Object.entries(p.url_patterns)81                          .sort((a, b) => b[1] - a[1])82                          .slice(0, 3)83                          .map(([k, v]) => `${k} (${v})`)84                          .join("  ")}85                      </td>86                      <td className="text-[11px]">87                        {Object.entries(p.entity_types)88                          .sort((a, b) => b[1] - a[1])89                          .slice(0, 4)90                          .map(([k, v]) => (91                            <Tag key={k} className="mr-1">92                              {k} {v}93                            </Tag>94                          ))}95                      </td>96                    </tr>97                  ))}98                </Table>99              </div>100            </Panel>101          </div>102          <Panel title="Runtime API map" sub="action → response shape → entity types (§71) — the learned frontend API of the platform" pad={false}>103            <div className="p-4">104              {rps.length ? (105                <Table head={["endpoint", "entity types", "triggered by", "seen", "yield", "confidence"]} dense>106                  {rps.map((r) => (107                    <tr key={r.shape_hash}>108                      <td className="mono text-[11.5px]">109                        <Tag color="#9d7bff" className="mr-1">110                          {r.method}111                        </Tag>112                        {r.hostname}113                        <span className="text-fg-2">{r.path_pattern}</span>114                        {r.graphql_operation && <Tag color="#ff7ad9" className="ml-1">{r.graphql_operation}</Tag>}115                        <div className="text-[10px] text-dim">#{r.shape_hash.slice(0, 10)}</div>116                      </td>117                      <td>118                        {Object.entries(r.likely_entity_types).map(([t, n]) => (119                          <Tag key={t} color="#38e1ff" className="mr-1">120                            {t} {n}121                          </Tag>122                        ))}123                      </td>124                      <td>125                        {Object.entries(r.triggered_by).map(([a, n]) => (126                          <Tag key={a} color="#ffb347" className="mr-1">127                            {a.toLowerCase()} {n}128                          </Tag>129                        ))}130                      </td>131                      <td className="mono">{r.observed_count}</td>132                      <td className="mono">{(r.entity_yield_total / Math.max(1, r.observed_count)).toFixed(1)} ent/resp</td>133                      <td className="w-32">134                        <Bar value={r.confidence} color={r.confidence >= 0.5 ? "#43e69a" : "#ffb347"} />135                        <div className="mono text-[10px] text-dim">{r.confidence.toFixed(2)}</div>136                      </td>137                    </tr>138                  ))}139                </Table>140              ) : (141                <Empty>no entity-bearing schema learned yet</Empty>142              )}143            </div>144          </Panel>145          <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">146            <Panel title="Navigation patterns" sub="action from a page type → where it leads and what it yields (§30)" pad={false}>147              <div className="p-4">148                <Table head={["action", "from", "→ to", "seen", "avg new ent", "avg resp", "fail"]} dense>149                  {aps.map((a) => (150                    <tr key={a.action_type + a.from_page_type}>151                      <td>152                        <Tag color="#ffb347">{a.action_type.toLowerCase()}</Tag>153                      </td>154                      <td>155                        <Tag color="#38e1ff">{a.from_page_type}</Tag>156                      </td>157                      <td className="text-[11px]">158                        {Object.entries(a.to_page_types)159                          .sort((x, y) => y[1] - x[1])160                          .map(([k, v]) => `${k} ${v}`)161                          .join(", ")}162                      </td>163                      <td className="mono">{a.observed_count}</td>164                      <td className="mono">{a.avg_new_entities.toFixed(1)}</td>165                      <td className="mono">{a.avg_network_responses.toFixed(1)}</td>166                      <td className="mono">{a.failures}</td>167                    </tr>168                  ))}169                </Table>170              </div>171            </Panel>172            <Panel title="Media delivery patterns" pad={false}>173              <div className="p-4">174                {Object.values(m.media_patterns).length ? (175                  <Table head={["host", "kind", "seen"]} dense>176                    {Object.values(m.media_patterns)177                      .sort((a, b) => b.observed_count - a.observed_count)178                      .map((mp) => (179                        <tr key={mp.hostname + mp.kind}>180                          <td className="mono text-[11.5px]">{mp.hostname}</td>181                          <td>182                            <Tag color="#ff7ad9">{mp.kind}</Tag>183                          </td>184                          <td className="mono">{mp.observed_count}</td>185                        </tr>186                      ))}187                  </Table>188                ) : (189                  <Empty>none yet</Empty>190                )}191              </div>192            </Panel>193          </div>194        </>195      )}196    </div>197  );198}199